package cn.darksoul3.spark.dataset

import org.apache.spark.rdd.RDD
import org.apache.spark.sql.types.{StringType, StructField, StructType}
import org.apache.spark.sql.{Row, SaveMode, SparkSession}

object WCWithDataFrameByTuple {

  def main(args: Array[String]): Unit = {

    val ss: SparkSession = SparkSession.builder().appName("WCWithDataSet").master("local[*]").getOrCreate()
    val sc = ss.sparkContext

    val lines: RDD[String] = sc.textFile("C:\\Users\\cary2\\Desktop\\wordcount.txt")

    import ss.implicits._
    val frame = lines.map(line => {
      val fields = line.split("\\s+")
      (fields(0))
    }).toDF("name")

    frame.write.mode(SaveMode.Overwrite).json("wc-json")
    ss.stop()
  }

}